package com.bigdata.core.transformations

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
 * repartition
 * 重新分区，可以将RDD的分区增多或者减少，会产生shuffle
 * coalesce(num,true) = repartition(num)
 * 用于增大下游分区
 * 结合mapPartitionWithIndex算子使用，观察分区内数据分配结果
 */
object Demo19_repartition {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local").setAppName("repartition")
    val sc = new SparkContext(conf)
    sc.setLogLevel("error")

    val rdd1: RDD[String] = sc.parallelize(List[String](
      "love1", "love2", "love3", "love4",
      "love5", "love6", "love7", "love8",
      "love9", "love10", "love11", "love12"), 3)

    val result: RDD[String] = rdd1.repartition(4)
    println(result.getNumPartitions)
    result.foreach(println)

  }
}
